mediawiki.rar
import requests
import json
from pathlib import Path
from tqdm import tqdm
from bs4 import BeautifulSoup
WEKEPEDIA_API_URL = "https://he.wikipedia.org/w/api.php"
WIKISOURCE_API_URL = "https://he.wikisource.org/w/api.php"
HAMICHLAL_API_URL = "https://www.hamichlol.org.il/w/api.php"
def get_authors_list(json_path: Path) -> set[str]:
all_authors = set()
with json_path.open('r', encoding='utf-8') as f:
data = json.load(f)
for book in data:
authors = book.get("heAuthors")
if authors is None:
continue
for author in authors:
if author and author.strip():
all_authors.add(author.strip())
return all_authors
def get_search_results(url: str, author_name: str, session: requests.Session, params: dict | None = None) -> requests.Response:
if params is None:
params = {}
headers = {
"User-Agent": "MyWikipediaScript/1.0 (your_email@example.com)"
}
params.update({
"action": "query",
"format": "json",
"list": "search",
"srsearch": author_name,
"utf8": 1
})
return session.get(url, params=params, headers=headers)
def prosess_response(response: requests.Response) -> list[dict]:
result = []
data = response.json()
for item in data['query']['search']:
snippet = BeautifulSoup(item['snippet'], "html.parser").get_text()
title = item['title']
page_id= item['pageid']
result.append({
"title": title,
"snippet": snippet,
"page_id": page_id
})
return result
def main():
all_results = {}
metadata_path = Path(r"C:\Users\User\Downloads\all_metadata.json")
# outpoot_path = Path(r"C:\Users\User\Downloads\authors_wikipedia_results.json")
# outpoot_path = Path(r"C:\Users\User\Downloads\authors_wikisource_results.json")
outpoot_path = Path(r"C:\Users\User\Downloads\authors_hamichlol_results.json")
authors = get_authors_list(metadata_path)
if outpoot_path.exists():
with outpoot_path.open("r", encoding="utf-8") as f:
all_results = json.load(f)
with requests.Session() as session:
for author in tqdm(authors, desc="Processing authors"):
if author in all_results:
continue
try:
# search_results = get_search_results(WEKEPEDIA_API_URL,author, session)
# search_results = get_search_results(WIKISOURCE_API_URL, author, session, {"srnamespace": 108})
search_results = get_search_results(HAMICHLAL_API_URL, author, session)
all_results[author] = prosess_response(search_results)
except Exception as e:
print(f"Error processing author {author}: {e}")
with outpoot_path.open('w', encoding='utf-8') as f:
json.dump(all_results, f, ensure_ascii=False, indent=4)
if __name__ == "__main__":
main()
מבוסס על שמות המחברים מהקובץ הזה.
צריך סינון של הדף הרלווטי (אולי ai), מה שמופיע שם זה תוצאות חיפוש, בויקיטקסט כל התוצאות הם ממרחב השם "מחברים".
